package com.sinotrust.classify.mba;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.Map.Entry;

import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.TermEnum;
import org.kernelab.basis.DataReader;
import org.kernelab.basis.DataWriter;
import org.kernelab.basis.Tools;


import test.IndexContainWords;
import test.SvmOutput;

public class DataOperation {
	public String positiveIndex;   //商业数据索引文件的名
	public String negativeIndex;	//非商业数据索引文件的名
	public String positiveData;      // 商业数据的向量文件名
	public String negativeData;		// 非商业数据的向量文件名
	public TreeSet<Term> jiaor = new TreeSet<Term>();    // 获取两种数据的词交集
	public TreeSet<Term> r1 = new TreeSet<Term>();     // 非商业数据的向量
	public TreeSet<Term> r2 = new TreeSet<Term>();    // 商业数据的向量
	
	double[] CoverageResult(IndexReader in, String keywordfile) throws IOException{
		double[] r =new double[in.numDocs()]; 
		int i =0;
		Set<IndexContainWords>  idr = indexReverse(in);
		for (IndexContainWords tmp: idr ){
			r[i++] = Coverage(tmp,keywordfile);
		}
		return r;
		
	}

	private double Coverage(IndexContainWords tmp, String keywordfile) throws IOException {
		double r = 0;
		BufferedReader br = new BufferedReader(new FileReader(keywordfile));
		Set<String> list = new HashSet<String>();
		String line;
		String[] words;
		while((line = br.readLine())!=null) {
			words = line.split("\t");
			list.add(words[0]);
		}
		for(String word : tmp.words){
			if(list.contains(word)){
				r++;
			}
		}
		return r/list.size();
	}

	Set<IndexContainWords> indexReverse(IndexReader ir) throws IOException {
		 Set<IndexContainWords> indexReverse = new TreeSet<IndexContainWords>();
		for (int i = 0; i < ir.numDocs(); i++) {
			IndexContainWords icw= new IndexContainWords();
			icw.id = i;
			Set<String> s = new HashSet<String>();
			icw.words = s;
			indexReverse.add(icw);
		}
		TermEnum termEnum = ir.terms();
		while (termEnum.next()) {
			TermDocs termDocs = ir.termDocs(termEnum.term());
			while (termDocs.next()) {
				String term = termEnum.term().text();
					for ( IndexContainWords  tmp : indexReverse) {
						if (tmp.id == termDocs.doc()) {
							tmp.words.add(term);							
						}
					}
			}
		}	
	 return indexReverse;
}
	void Out(IndexReader ir, TreeSet<Term> jiaor, int index, String out)
			throws CorruptIndexException, IOException {
		double value = 0;
		Map<Integer, SvmOutput> setSvmOutput = new HashMap();
		for (int i = 0; i < ir.numDocs(); i++) {
			SvmOutput tmp = new SvmOutput();
			for (Term tmp1 : jiaor) {
				tmp.relation.put(tmp1.term, value);
			}
			setSvmOutput.put(i, tmp);
		}
		Set<String> s = new HashSet<String>();
		for (Term tmp : jiaor) {
			s.add(tmp.term);
		}
		TermEnum termEnum = ir.terms();
		while (termEnum.next()) {
			TermDocs termDocs = ir.termDocs(termEnum.term());
			while (termDocs.next()) {
				String term = termEnum.term().text();
				if (s.contains(term)) {
					float tf = termDocs.freq();
					for (Entry<Integer, SvmOutput> tmp : setSvmOutput
							.entrySet()) {
						if (tmp.getKey() == termDocs.doc()) {
							for (Entry<String, Double> tmp1 : tmp.getValue().relation
									.entrySet()) {
								if (tmp1.getKey().equals(term)) {
									tmp1.setValue((double) tf);
								}
							}
						}
					}

				}
			}
		}
		DataWriter dw = new DataWriter();
		dw.setDataFile(new File(out));
		dw.setAutoFlush(false);
		Set<Entry<Integer, SvmOutput>> set = setSvmOutput.entrySet();
		for (Map.Entry<Integer, SvmOutput> tmp : set) {
			int i = 1;
			dw.print(index + " ");
			for (Entry<String, Double> tm : tmp.getValue().relation.entrySet()) {
				dw.print(i + ":" + tm.getValue() + " ");
				i++;
			}
			dw.write(" ");
		}
		ir.close();
		dw.close();

	}

	public void freStatic() throws IOException {
		r1 = FreStatic(negativeIndex);
		r2 = FreStatic(positiveIndex);
	}

	public void resultWriter() throws CorruptIndexException, IOException {
		featureBring(negativeIndex, jiaor, -1, negativeData);// 生成对应交集的特征向量
		featureBring(positiveIndex, jiaor, 1, positiveData);// 生成对应交集的特征向量
		DataOperation.mergeFile("negative.txt", "positive.txt", "dataSvm.txt");// 合并正负数据集合
		Tools.debug("分类器已经生成");
	}

	public void jiao() throws FileNotFoundException {
		jiaor = jiao(r1, r2);
		complemnt(r1,"NonCommerce.txt");
		complemnt(r2,"Commerce.txt");
		// Tools.debug(jiaor);
	}

	private void complemnt(TreeSet<Term> r12,String out) throws FileNotFoundException {
		// TODO Auto-generated method stub
		TreeSet<Term> r = new TreeSet<Term>();
		DataWriter dw = new DataWriter();
		dw.setDataFile(new File(out));
		dw.setAutoFlush(false);
		Set<String> s = new HashSet<String>();
		for (Term tmp : jiaor) {
			s.add(tmp.term);
		}
		// int i = 0;
		for (Term tm1 : r12) {
				if (!s.contains(tm1.term))
					dw.write(tm1.term+"\t"+tm1.value);
		}
		dw.close();
	}

	private TreeSet<Term> jiao(TreeSet<Term> r1, TreeSet<Term> r2)
			throws FileNotFoundException {
		TreeSet<Term> r = new TreeSet<Term>();
		TreeSet<String> s = new TreeSet<String>();

		String out = "jiaoword.txt";
		DataWriter dw = new DataWriter();
		dw.setDataFile(new File(out));
		dw.setAutoFlush(false);

		// int i = 0;
		for (Term tm1 : r1) {
			for (Term tm2 : r2) {
				if (tm1.term.equals(tm2.term)) {
					Term tmp = new Term();
					tmp.term = tm1.term;
					tmp.value = tm1.value - tm2.value;
					// Tools.debug(tm1.term + "\t" + tm2.term + "\t" + tm1.value
					// + "\t" + tm2.value);
					r.add(tm1);
					s.add(tmp.term);
					dw.write(tmp.term);
				}

			}
		}
		dw.close();
		return r;

	}

	private TreeSet<Term> FreStatic(String index) throws IOException {
		// TODO Auto-generated method stub
		IndexReader ir = IndexReader.open(index);
		int n = ir.numDocs();
		TermEnum termEnum = ir.terms();
		TreeSet<Term> result = new TreeSet<Term>();
		while (termEnum.next()) {
			TermDocs termDocs = ir.termDocs(termEnum.term());
			// Tools.debug(termDocs);
			while (termDocs.next()) {
				// if termEnum.term().text() in List<term>,termDocs.freq();
				String term = termEnum.term().text();
				boolean keywordFlag = isKeywordFlag(term);
				float tf = termDocs.freq();
				float df = termEnum.docFreq();
				
				double weight = tf * Math.log(n / df + 0.01);//
				
				if (weight > 1 && keywordFlag) {
					// System.out.print(term+"\t"+weight+"\t"+keywordFlag+"\n");
					Term tmp = new Term();
					tmp.term = term;
					tmp.value = df;
					result.add(tmp);
					// System.out.println("term:"+termEnum.term().text()+";weight:"+weight);
				}
			}
		}
		ir.close();
		return result;
	}

	private boolean isKeywordFlag(String term) {
		boolean tmp = true;
		for (int i = 0; i < term.length(); i++) {
			if ((term.charAt(i) >= 'a' && term.charAt(i) <= 'z')
					|| (term.charAt(i) >= 'A' && term.charAt(i) <= 'Z')
					|| (term.charAt(i) >= '0' && term.charAt(i) <= '9')) {
				tmp = false;
				break;
			}
		}
		return tmp;
		// TODO Auto-generated method stub
	}

	public static void featureBring(String string, TreeSet<Term> jiaor,
			int index, String out) throws CorruptIndexException, IOException {
		double value = 0;
		// Tools.debug(jiaor);
		Map<Integer, SvmOutput> setSvmOutput = new HashMap();
		IndexReader ir = IndexReader.open(string);
		//
		for (int i = 0; i < ir.numDocs(); i++) {
			SvmOutput tmp = new SvmOutput();
			for (Term tmp1 : jiaor) {
				// Tools.debug(tmp1.key + "\t" + value);
				// Tools.debug(tmp.relation);
				tmp.relation.put(tmp1.term, value);
			}
			setSvmOutput.put(i, tmp);
		}
		Set<String> s = new HashSet<String>();
		for (Term tmp : jiaor) {
			s.add(tmp.term);
		}
		TermEnum termEnum = ir.terms();
		while (termEnum.next()) {
			TermDocs termDocs = ir.termDocs(termEnum.term());
			while (termDocs.next()) {
				String term = termEnum.term().text();
				if (s.contains(term)) {
					float tf = termDocs.freq();
					for (Entry<Integer, SvmOutput> tmp : setSvmOutput
							.entrySet()) {
						if (tmp.getKey() == termDocs.doc()) {
							for (Entry<String, Double> tmp1 : tmp.getValue().relation
									.entrySet()) {
								if (tmp1.getKey().equals(term)) {
									tmp1.setValue((double) tf);
								}
							}
						}
					}

				}
			}
		}
		DataWriter dw = new DataWriter();
		dw.setDataFile(new File(out));
		dw.setAutoFlush(false);
		Set<Entry<Integer, SvmOutput>> set = setSvmOutput.entrySet();
		for (Map.Entry<Integer, SvmOutput> tmp : set) {
			int i = 1;
			double sum = sum(tmp.getValue());
			// System.out.print(index + " ");
			dw.print(index + " ");
			// dw.print(tmp.getKey() + " ");
			for (Entry<String, Double> tm : tmp.getValue().relation.entrySet()) {
				// System.out.print(i + ":" + tm.getValue() + " ");
				if(sum>0){
					dw.print(i + ":" + tm.getValue() + " ");
				}
				// dw.print(i + ":" +tm.getKey()+ tm.getValue() + " ");
				i++;
			}
			// Tools.debug(" ");
			dw.write(" ");
		}
		ir.close();
		dw.close();
		// return result;

	}

	private static double sum(SvmOutput svmOutput) {
		// TODO Auto-generated method stub
		double s = 0;
		for (Map.Entry<String, Double> sum : svmOutput.relation.entrySet()){
			s=s+sum.getValue();
		}
		return s;
	}

	public static void mergeFile(String string, String string2, String string3)
			throws IOException {
		// TODO Auto-generated method stub
		BufferedReader br = new BufferedReader(new FileReader(string));
		DataWriter dw = new DataWriter();
		dw.setDataFile(new File(string3));
		dw.setAutoFlush(false);
		String tmp;
		// Tools.debug(tmp);
		while ((tmp = br.readLine()) != null) {
			// Tools.debug(tmp);
			dw.write(tmp);
		}

		br = new BufferedReader(new FileReader(string2));
		// tmp = br.readLine();
		while ((tmp = br.readLine()) != null) {
			dw.write(tmp);
		}
		br.close();
		dw.close();

	}

}
